package jpave.annotator;
/* 
 * There will only be one DoBlast object 
 * 
 * annotatorPAVE -q causes CoreMain.yesno to return true on all prompts
 * 	without actually doing the prompt
 */
import java.io.*;
import java.util.HashMap;
import java.util.ArrayList;

import util.methods.FileHelpers;
import util.methods.TimeHelpers;
import util.methods.BlastArgs;
import util.Debug.ErrorReport;
import jpave.annotator.DoUniProt;
import jpave.annotator.CoreMain;
import jpave.annotator.LineParser;
import jpave.database.CoreDBWrapper;
import jpave.dataholders.BlastHitData;
import jpave.manager.EditAnnoPanel;

public class DoBlast {
	private boolean debug = false;
	private String[] ntFormatFiles = { ".nin", ".nhr",  ".nsq"};
	private String[] aaFormatFiles = { ".phr", ".pin", ".psq"};
	private String[] diaFormatFiles = {".dmnd"};
	private String[] uFormatFiles = {".udb"};
	final int maxHitIDlen = 30; // hard coded in Schema.java
		
	public void setMainObj(DoUniProt db, CoreAnnotator a, CoreDBWrapper s) {
		uniObj = db;
		annoObj = a;
		sqlObj = s;
	}
	
	final boolean doPrompt=true;
	final boolean noPrompt=false;
	
	// called from CoreMain to get state
	public boolean doPairs() { 
		if (doSelfBlast || doTSelfBlast) return true;
		if (selfBlastFile != null) return true;
		if (tselfBlastFile != null) return true;
		return false;
	}
	public int numDB() { return dbInfo.size(); }
	
	/*********************************************************
	 * Checks all supplied files and sets dbType, isProtein and add Blast program type to ARGs
	 * Checks for existing blast and ask user whether to use the existing file
	 * 		If ReUse, only print warnings
	 */
	public boolean testFilesAndcreateOutFiles() {
		CoreMain.Print("\nChecking blast and annoDB fasta files");

		// check to see if the databases have been processed before
		if (!testDatabaseTable()) return false;
	
		// check user supplied blast files and dbfile. Also sets dbType
		int cntErr=0; 
		doDBblast = false;  
		
		for (int i=0; i < dbInfo.size(); i++) {
			if (dbInfo.get(i).doBlast) {
				if (!testExistString(dbInfo.get(i).DBfastaFile))
					CoreMain.die("TCW error: fasta name must exist for DB#" + i);
				
				boolean found=false;
			 	File f = new File(dbInfo.get(i).DBfastaFile);
			 	if (f.isFile() && f.exists()) {
			 		found=true;
			 		dbInfo.get(i).fileSize = f.length();
			 	}
			 	else {
			 		f = new File(dbInfo.get(i).DBfastaFile + ".gz");
			 		if (f.isFile() && f.exists()) {
			 			found=true;
				 		dbInfo.get(i).fileSize = f.length();
				 		dbInfo.get(i).DBfastaFile += ".gz";
			 		}
			 	}
				if (!found) {
					CoreMain.PrtError("File does not exist: " + dbInfo.get(i).DBfastaFile);
					cntErr++; 
				}
				else {
					if (testDBfastaFileAndSetSearchPgm(i)) doDBblast = true; // Prints file name
					else cntErr++;
				}
			}
			else {
				if (!testExistString(dbInfo.get(i).tabularFile))
					CoreMain.die("TCW error: blast file must exist for DB#" + i);
				
				String label = "DB#"+i;
				boolean b = false;
				if (testExistFile(dbInfo.get(i).tabularFile)) 
					if (testTabularFile(label, dbInfo.get(i).tabularFile, i, doPrompt)) {
						CoreMain.PrtSpMsg(1, (i+1) + ". Load file " + 
								CoreMain.removeCurrentPath(dbInfo.get(i).tabularFile));
						b = true;
					}
				if (b) { // optional DB file
					if (testExistString(dbInfo.get(i).DBfastaFile)) {
						if (FileHelpers.fileExists(dbInfo.get(i).DBfastaFile)) {
							if (!testDBfastaFileAndSetSearchPgm(i)) b = false;
							else if (!testFastaLtBlastDates(i, doPrompt)) b = false;
						}
						else {
							b = false;
							CoreMain.PrtError("DBfasta does not exist " + dbInfo.get(i).DBfastaFile);
						}
					}
				}
				if (!b) cntErr++;
			}
		}
		
		if (!doSelfBlast && testExistString(selfBlastFile)) {
			if (!testExistFile(selfBlastFile)) cntErr++;
			else if (!testTabularFile("Selfblast", selfBlastFile, -1, doPrompt)) cntErr++;
			else CoreMain.PrtSpMsg(2, "Load pair file " + CoreMain.removeCurrentPath(selfBlastFile));
		}
		else if (doSelfBlast) CoreMain.PrtSpMsg(1, "Create pairs self blast");
		if (!doTSelfBlast && testExistString(tselfBlastFile)) {
			if (!testExistFile(tselfBlastFile)) cntErr++; 
			else if (!testTabularFile("TselfBlast", tselfBlastFile, -1, doPrompt) ) cntErr++;
			else CoreMain.PrtSpMsg(2, "Load pair file " + CoreMain.removeCurrentPath(tselfBlastFile));
		}
		else if (doTSelfBlast) CoreMain.PrtSpMsg(1, "Create pairs translated self blast");
		
		if (cntErr>0) {
			CoreMain.PrtError("Failed checking files and databases\n");
			return false;
		}
				
		if (!(doSelfBlast || doTSelfBlast|| doDBblast)) {
			CoreMain.PrtSpMsg(0,"No blasts to be executed - complete check files\n");
			return true;
		}
		
		/**** create blast output file names and see if they already exist ****/
		
		doDBblast = false;
		for (int ix=0; ix < dbInfo.size(); ix++) {
			if (! dbInfo.get(ix).doBlast) continue; // already have blast file
			
			// create blast file name
			String tt =  dbInfo.get(ix).typeTaxo;
			int k=0;
			for (int j=0; j< ix; j++) 
				if (dbInfo.get(j).typeTaxo.equals(tt)) k++;			
			if (k>0) tt = tt + "_" + k;
									
			if (dbInfo.get(ix).doSubset)
				dbInfo.get(ix).tabularFile  = uniBlastPrefix + tt + "_sub.tab";
			else 
				dbInfo.get(ix).tabularFile  = uniBlastPrefix + tt  + ".tab";
			
			if (testUseExistingFile(ix, dbInfo.get(ix).tabularFile))  // exist and reuse?
				dbInfo.get(ix).doBlast = false;
			else doDBblast = true; 
		}
		
		if (doSelfBlast) {
			selfBlastFile = uniBlastPrefix + "blast_self.tab";
			if (testUseExistingFile(-1, selfBlastFile))  doSelfBlast = false;
		}
		if (doTSelfBlast) {
			tselfBlastFile = uniBlastPrefix + "blast_tself.tab";
			if (testUseExistingFile(-2, tselfBlastFile))  doTSelfBlast = false;
		}
		if (!(doSelfBlast || doTSelfBlast|| doDBblast))  // using existing files
			CoreMain.PrtSpMsg(0,"No blasts to be executed - complete check files\n");
		else
			CoreMain.PrtSpMsg(0,"Complete check files\n");
		return true;
	}
		
	private boolean testExistString(String theFile) {		
		if (theFile == null || theFile.equals("") || theFile.equals("-")) 
			return false;
		return true;
	}
	
	private boolean testExistFile(String theFile) {				
		if (!FileHelpers.fileExists(theFile)) {
			CoreMain.PrtError(" Cannot locate the file '" + theFile + "'");
			return false;
		} 
		return true;
	}
	
	private boolean testUseExistingFile(int ix, String file) {
		if (! FileHelpers.fileExists(file)) {
			file = file + ".gz"; // CAS 2/10/15
			if (! FileHelpers.fileExists(file)) return false;
			else dbInfo.get(ix).tabularFile = file;
		}
		if (CoreMain.noPrompt) return true;
		
		String msg="";
		if (ix == -1) msg = "Self blast";
		else if (ix == -2) msg = "Tself blast";
		else msg = "DB#" + (ix+1) + " blast";
		if (!testTabularFile(msg, file, -1, noPrompt)) return false;

		File f = new File(file);
		String s = TimeHelpers.longDate(f.lastModified());
		boolean ans=false;
		
		CoreMain.PrtSpMsg(1, msg);
		if (ix >=0) 
			CoreMain.PrtSpMsg(2, "DBfasta file:  " + dbInfo.get(ix).DBfastaNoPath);

		CoreMain.PrtSpMsg(2,     "Output exists:  " + CoreMain.removeCurrentPath(file) + "; Dated: " + s);
		ans = CoreMain.yesNo("Yes = use this existing file; No = perform new blast; ");
		
		if (ans && ix >= 0) ans = testFastaLtBlastDates(ix, true);
		return ans;
	}
	
	private boolean testFastaLtBlastDates(int ix, boolean prompt) {
		String fasta = dbInfo.get(ix).DBfastaFile;
		String blast = dbInfo.get(ix).tabularFile;
		
		File ff = new File(fasta);
		long fastaDate = ff.lastModified();
		
		File bf = new File(blast);
		long blastDate = bf.lastModified();
		
		if (fastaDate <= blastDate) return true;
		
		String sf = TimeHelpers.longDate(fastaDate);
		String sb = TimeHelpers.longDate(blastDate);
		String msg = "Date for DBfasta file is greater than blast output file\n" +
				"DBfasta: " + sf + " " + CoreMain.removeCurrentPath(fasta) + "\n" +
				"Output : " + sb + " " + CoreMain.removeCurrentPath(blast);
	
		if (!prompt) return false;
		
		CoreMain.PrtWarn(msg);
		boolean ans = CoreMain.yesNo("Use this file anyway (y) or do blast (n): ");
		if (ans) return true;
		return false;
	}
	
    private boolean testTabularFile (String msg, String blastFile, int ix, boolean prompt) 
	{
		try {
			// Get the first non-empty line
			BufferedReader reader = CoreMain.openGZIP(blastFile);
			String line = reader.readLine();
			if (line != null) {
				line = line.trim();
				while ( line != null 
						&& (line.length() == 0 || line.charAt(0) == '#') )
							line = reader.readLine().trim();
			}
			reader.close();
			
			boolean ans;
			if ( line == null ) {
				if (!prompt) return false;
				ans = CoreMain.yesNo(msg + " file " + blastFile + " is empty.  Continue anyway?");
				if (ans) return true;
				else return false;
			}

	    		String[] tokens = line.split("\t");
	    		if (tokens == null || tokens.length < 11) {
				if (prompt) CoreMain.PrtError(msg + " file " + blastFile + " is not -m 8 format");;
				return false;
	    		}
			
			if (ix == -1) return true; // not selfblast file
			
			// see if type is UniProt	
			LineParser lp = new LineParser();
			if (!lp.parseLine(line)) {
				CoreMain.PrtError("Cannot parse DBblast file: " + blastFile + "\n   Line:" + line);
				return false;
			}
			String type = lp.getDBtype();
			
			if (!dbInfo.get(ix).dbType.equals("")) {// already added by testDBfile
				if (!dbInfo.get(ix).dbType.equals(type)) {
					if (prompt) 
						CoreMain.PrtError("DB and blast file are different types:" +
								"\n   " + type + ":" + blastFile + 
								"\n   "	+ dbInfo.get(ix).dbType + ":" + dbInfo.get(ix).DBfastaFile);
					return false;
				}
			}
			setType(ix, type);
			dbInfo.get(ix).isProtein = true;
	
			return true;
		} 
		catch ( Exception err ) {
			ErrorReport.reportError(err, "Annotator - loading  Blast file");
			return false;
		} 
	}
	
	private boolean testDBfastaFileAndSetSearchPgm (int ix) {
		String fileName = dbInfo.get(ix).DBfastaFile;
		boolean dob = dbInfo.get(ix).doBlast;
		String prtFileName = CoreMain.removeCurrentPath(fileName);
		
		try {
			BufferedReader reader = CoreMain.openGZIP(fileName);
			String line1 = reader.readLine();
			LineParser lp = new LineParser ();
			
			if (line1 != null) {
				line1 = line1.trim();
				while ( line1 != null && (line1.length() == 0 || line1.charAt(0) == '#') )
					line1 = reader.readLine().trim();
			}
			if ( line1 == null ) { // must come after checking for a file of empty lines
				CoreMain.PrtError(prtFileName + ": the file " + prtFileName + " is empty." );
				reader.close();
				return false;
			}	
			if (!line1.startsWith(">")) {
				CoreMain.PrtError(prtFileName + ": fasta files should have '>' in column 0 of first non-blank line. ");
				reader.close();
				return false;
			}
			if (! lp.parseLine(line1)) {
				CoreMain.PrtError(prtFileName + ": cannot parse DBfasta file\n   Line:" + line1);
				return false;
			}
				
			// XXX determine blast program to use
			// blastx nucleotide against protein; translated nucleotide in 6 frames
			// blastp protein against protein
			// megablast or blastn nucleotide against nucleotide; no translation
			// could provide tblastx, where they both are translated.
			// not-valid protein against nucleotide; could do with tblastn
			String line2 = reader.readLine().trim();
			reader.close();
			if (line2 == null) {
				CoreMain.PrtError( "The file " + prtFileName + " is incorrect." );
				reader.close();
				return false;
			}
			String dbtype = lp.getDBtype();
			String seqtype = "pr";
			if (!dbtype.equals("sp") && !dbtype.equals("tr")) {
				if (BlastArgs.isNucleotide(line2)) seqtype="nt"; 
			}
			if (dbtype.equals("")) dbtype=seqtype;
			
			boolean PEPTIDE=sqlObj.isProteinDB();
			String action="", pgm="blast";
			String searchPgm=dbInfo.get(ix).searchPgm;
			if (searchPgm.equals(EditAnnoPanel.SEARCH_PGM_DEFAULT)) searchPgm="";
			boolean isZIP = (fileName.endsWith(".gz")) ? true : false;
			
			if (dbtype.equals("sp") || dbtype.equals("tr") || seqtype.equals("pr")) {
				dbInfo.get(ix).isProtein = true;
				if (PEPTIDE) action = "blastp";
				else action =  "blastx";
				
				if (searchPgm.equals("")) {
					if (dbtype.equals("sp")) pgm = "blast";
					else if (dbtype.equals("tr") || dbInfo.get(ix).fileSize>=1000000000L) {
						if (BlastArgs.isDiamond()) pgm = "diamond";
						else if (BlastArgs.isUsearch()) pgm="usearch";
						else pgm = "blast";
					}
					else pgm = "blast";
					
					if (isZIP && BlastArgs.isDiamond()) pgm = "diamond"; // takes precedence
					dbInfo.get(ix).blastArgs="";
				} 
				else pgm = searchPgm;
				
				if (dob) {
					String x = (ix+1) + ". " + pgm;
					if (dbtype.equals("gi")) CoreMain.PrtSpMsg(1, x + " Genbank Protein Fasta File: " + prtFileName);
					else if (dbtype.equals("sp")) CoreMain.PrtSpMsg(1, x + " SwissProt Fasta File: " + prtFileName);
					else if (dbtype.equals("tr")) CoreMain.PrtSpMsg(1, x + " Trembl Fasta File: " + prtFileName);
					else CoreMain.PrtSpMsg(1, x + " Protein Fasta File: " + prtFileName);
				}
			}
			else if (dbtype.equals("nt") || seqtype.equals("nt")){
				if (PEPTIDE) {
					CoreMain.PrtError(prtFileName + ": cannot have a nucleotide annoDB with a protein sTCW");
					return false;
				}
				
				dbInfo.get(ix).isProtein = false;
				action =  "blastn";
				if (!searchPgm.equals("blast")) {
					dbInfo.get(ix).blastArgs="";
					if (searchPgm.equals("diamond") || searchPgm.equals("usearch"))
						CoreMain.PrtWarn("Cannot use " + searchPgm + " with nucleotide annoDB -- using blast");
					pgm = "blast";
				}
				if (dob) {
					String x = (ix+1) + ". " + action;
					if (dbtype.equals("gi")) CoreMain.PrtSpMsg(1, x + " Genbank Nucleotide Fasta File: " + prtFileName);
					else CoreMain.PrtSpMsg(1, x + " Nucleotide Fasta File: " + prtFileName);
				}
			}
			else {
				CoreMain.PrtError("Could not determine type of file for " + prtFileName + 
						" - you may need to run blast and provide the tabular file for input.");
				return false;
			}
			if (isZIP && !pgm.equals("diamond")) {
				CoreMain.PrtError("The file " + prtFileName + " is compressed, which will fail with " + pgm);
				return false;
			}
			if (!dbInfo.get(ix).dbType.equals("")) { // already added by testBlastFile
				if (!dbInfo.get(ix).dbType.equals(dbtype)) {
					CoreMain.PrtError("DB and blast file are different types:" +
						"\n   " + dbtype + ":" + dbInfo.get(ix).DBfastaFile + 
						"\n   "	+ dbInfo.get(ix).dbType + ":" + dbInfo.get(ix).tabularFile);
					return false;
				}
			}
			else setType(ix, dbtype);
				
			dbInfo.get(ix).searchPgm=pgm;
			dbInfo.get(ix).exec = action;
			if (dbInfo.get(ix).blastArgs.equals("")) {
				if (pgm.equals("diamond")) dbInfo.get(ix).blastArgs = BlastArgs.getDiamondOptions();
				else if (pgm.equals("usearch")) dbInfo.get(ix).blastArgs = BlastArgs.getUsearchOptions();
				else {
					if (pgm.equals("blastx")) 
			    			dbInfo.get(ix).blastArgs = BlastArgs.getBlastxOptions();
					else if (pgm.equals("blastn")) 
			    			dbInfo.get(ix).blastArgs = BlastArgs.getBlastnOptions();
					else
						dbInfo.get(ix).blastArgs = BlastArgs.getBlastpOptions();
				}
			}
			return true;
		} 
		catch ( Exception err ) {
			ErrorReport.reportError(err, "Annotator - determining type of file " + prtFileName);
			return false;
		} 
	}
	
	private boolean testDatabaseTable () {
		// pja_database is not cleared until after all checks
		if (CoreMain.bDelAnno) return true;
		try {
			ArrayList <String> paths = sqlObj.loadDatabases();
			for (int i=0; i < paths.size(); i++) {
				for (int j=0; j < dbInfo.size(); j++) {
					if (paths.get(i).equals(dbInfo.get(j).tabularFile) ||
						paths.get(i).equals(dbInfo.get(j).DBfastaFile)) 
					{
						String str = "DB#" + dbInfo.get(j).dbNum + " The database " + 
								CoreMain.removeCurrentPath(paths.get(i)) + 
							"\n      has been processed previously. Continue?";
						boolean ans = CoreMain.yesNo(str);
						if (ans==false) return false;
					}					
				}
			}
			return true;
		}
		catch (Exception err) {
			ErrorReport.reportError(err, "checking existing annoDB files in the sTCW database");
			return false;
		}
	}
	/*************************************************
	 * Execute megablast (or blastp for proteins) and tblastx
	 *******************************************/
	public boolean runSelfBlast(String blastProgram) 
	{
		String outFile;
		String blastCmd;
		String blast, args;
	    writeUniTransFile(false); // creates unitrans.fasta (sets uniTransFile, so do before getBlastCmd)
	    boolean PEPTIDE=sqlObj.isProteinDB();
	    
		if (blastProgram.equals("tblastx")) {
			if (!doTSelfBlast) return true;
			if (PEPTIDE) {
			    CoreMain.PrtWarn("Cannot run translated self blast on a protein database");
			    return true;
			}
			blast = BlastArgs.getTblastxExec();
			args = BlastArgs.getTblastxOptions();
		    outFile = tselfBlastFile;
			blastCmd = BlastArgs.getTblastxCmd(uniTransFile, uniTransFile, 
			            outFile, CoreMain.getCPUs());
			CoreMain.StartTime("\n   Running Sequence translated selfblast ");
		}
		else {
			if (!doSelfBlast) return true;
			
			outFile = selfBlastFile;
			if (PEPTIDE) {
			    blast = BlastArgs.getBlastpExec();
	            args = BlastArgs.getBlastpOptions();
			    blastCmd = BlastArgs.getBlastpCmd(uniTransFile, uniTransFile, 
	                    outFile, CoreMain.getCPUs());
			}
			else {
			    blast = BlastArgs.getBlastnExec();
	            args = BlastArgs.getBlastnOptions();
			    blastCmd = BlastArgs.getBlastnCmd(uniTransFile, uniTransFile, 
                    outFile, CoreMain.getCPUs());
			}
			CoreMain.StartTime("\n   Running Sequence selfblast ");
		}				
		
		CoreMain.PrtSpMsg(3,"Execute:     " + blast);
		CoreMain.PrtSpMsg(3,"Parameters:  " + args);
		CoreMain.PrtSpMsg(3,"OutFile:     " + CoreMain.removeCurrentPath(outFile));
	
		try {
			boolean fmtFlag; 
			if (PEPTIDE) fmtFlag = runFormatDB("blast", uniTransFile, aaFormatFiles, "aa");
			else fmtFlag = runFormatDB("blast", uniTransFile, ntFormatFiles, "nt");
			
			// Run megablast or tblastx		
			Process p = Runtime.getRuntime().exec(blastCmd);
			p.waitFor();
			if (p.exitValue() != 0) {
			    CoreMain.PrtSpMsg(1, blastCmd);
				CoreMain.PrtError("blast failed with exit value = " + p.exitValue());
				return false;
			}

			//if (fmtFlag) removeFormatDB(uniTransFile, exFiles);
			CoreMain.EndTime("   Complete " + blast + " self-blast");

			sqlObj.reset(); // times out during blast			
		} catch (Exception e) {
			CoreMain.PrtError("Failed command: " + blastCmd);
		    if (BlastArgs.isBlastPlus()) {
		         CoreMain.PrtSpMsg(3, "+++ legacy blast databases are not compatiable with blast+");
		         CoreMain.PrtSpMsg(3, "     You may need to remove legacy formated files");
		    }
			ErrorReport.reportError(e, "Executing blast");
			return false;
		}
		return true;
	}
	/*************************************************
	 * Execute all unitrans against DBs
	 *******************************************/
	public String getRunDbBlastFile(int i) {
		
		if (dbInfo.get(i).doBlast) {
			if (!runDBblast(i)) return null;
			try {
				sqlObj.reset(); // times out during blast
			}
			catch (Exception e) {} // dies if fails
		}
		return dbInfo.get(i).tabularFile;
	}
	private boolean runDBblast(int ix)
	{
		if (!BlastArgs.blastExists()) {
			System.err.println("Error: No blast program detected or identified in HOSTS.cfg");
			return false;
		}
		String dbFile = dbInfo.get(ix).DBfastaFile;
		String blastOp = dbInfo.get(ix).blastArgs;
		String outFile = dbInfo.get(ix).tabularFile;
		String action = dbInfo.get(ix).exec;
		String pgm = dbInfo.get(ix).searchPgm;
		boolean isProteinDB = dbInfo.get(ix).isProtein;
    		long totalTime = TimeHelpers.getTime();
    	
    		String file = dbInfo.get(ix).DBfastaNoPath;
    		String size="";
    		if (dbInfo.get(ix).fileSize>=1000000000) {
    			size = String.format("%.0fGb", (float) ((double)dbInfo.get(ix).fileSize)/1000000000.0);
    		}
    		else if (dbInfo.get(ix).fileSize>=1000000) {
    			size = String.format("%.0fMb", (float) ((double)dbInfo.get(ix).fileSize)/1000000.0);
    		}
    		else if (dbInfo.get(ix).fileSize>=1000) {
    			size = String.format("%.0fkb", (float) ((double)dbInfo.get(ix).fileSize)/1000.0);
    		}
    		else size = String.format("%d", (int) dbInfo.get(ix).fileSize);
    		
		CoreMain.PrtSpMsg(2, "DB#" + dbInfo.get(ix).dbNum + " " + file  + " " +
				size + "       " + TimeHelpers.getDate());

		// creates file of sequences on the first call to this
		if (! writeUniTransFile(dbInfo.get(ix).doSubset)) 
				CoreMain.die("execAnno aborted - error creating sequence file for blast");
		
		// get program name
		String type = "aa", blastCmd="";
		if (action.equals("tblastx") || action.equals("blastn")) type = "nt"; 
		
		if (pgm.equals("diamond")) {
			blastCmd = 
				BlastArgs.getDiamondCmd(uniTransFile, dbFile, outFile, action, blastOp, CoreMain.getCPUs());
			if (!checkDiamondtmpDir(blastOp)) return false;
			CoreMain.PrtSpMsg(3,"Use:     " + pgm + " "+ blastOp + " --threads " + CoreMain.getCPUs());
		}
		else if (pgm.equals("usearch")) {
			blastCmd = 
				BlastArgs.getUsearchCmd(uniTransFile, dbFile, outFile, blastOp);
			CoreMain.PrtSpMsg(3,"Use:     " + pgm + " "+ blastOp);
		}
		else if (pgm.equals("blast")) {
			blastCmd = 
		        BlastArgs.getBlastCmd(uniTransFile, dbFile, outFile, action, blastOp, CoreMain.getCPUs());
			CoreMain.PrtSpMsg(3,"Use:     " + pgm + " " + blastOp + " -num_threads "  + CoreMain.getCPUs());;
		}
		CoreMain.PrtSpMsg(3,     "OutFile: " + CoreMain.removeCurrentPath(outFile));
				
		if (action==null || action.equals("")) {
			System.err.println("Error: No blast program detected or identified in HOSTS.cfg");
			return false;
		}
		
		try {
			// CAS 1/30/15 - how did it lose the check for isProteinDB? it was there...
			boolean goodFmt=true;
			if (isProteinDB) {
				if (pgm.equals("diamond")) goodFmt = runFormatDB(pgm, dbFile, diaFormatFiles, type);
				else if (pgm.equals("usearch")) goodFmt = runFormatDB(pgm, dbFile, uFormatFiles, type);
				else goodFmt = runFormatDB(pgm, dbFile, aaFormatFiles, type);
			}
			else goodFmt = runFormatDB(pgm, dbFile, ntFormatFiles, type);			
			if (!goodFmt) return false;
			
			Process p = Runtime.getRuntime().exec(blastCmd);
			p.waitFor();
			if (p.exitValue() != 0) {
				CoreMain.PrtError("blast failed with exit value = " + p.exitValue());
				CoreMain.Print("Command: " + blastCmd);
				return false;
			}
			
			CoreMain.PrtSpMsgTime(3, "DB blast completed", totalTime);
			
			// XXX ZZZ diamond should not compress output unless asked, but I think it is.
			if (!new File(outFile).exists()) {
				if (!new File(outFile + ".gz").exists()) 
					CoreMain.PrtWarn("No output file created: " + CoreMain.removeCurrentPath(outFile));
				return false;
			}
			else return true;
		} catch (Exception e) {
		    CoreMain.Print("Failed Command: " + blastCmd);
			ErrorReport.reportError(e, "Executing search");
			return false;
		}
	}
	// used by both selfblasts and DB blasts
	private boolean runFormatDB (String pgm, String dbFileName, String[] checkFiles, String type) {
		String formatdbCmd;
		long dbFastaTime = (new File(dbFileName)).lastModified();
			
		try {
			boolean exists = true;
			boolean old = false;
			
			for (int i = 0; i < checkFiles.length && exists; i++) {
				String s1 = dbFileName + checkFiles[i];
				String s2 = dbFileName + ".00" + checkFiles[i];  // BLAST specific
				if (!FileHelpers.fileExists(s1) && !FileHelpers.fileExists(s2)) {
					exists=false;
					break;
				}
			
				if (!FileHelpers.fileExists(s1)) s1 = s2;
				long fdbTime = (new File(s1)).lastModified();
				
				if (fdbTime < dbFastaTime) {
					CoreMain.PrtSpMsg(3,"Format files are out of date - reformatting");
					old = true;
					break;
				}				
			}
			if (exists && !old) {
				CoreMain.PrtSpMsg(3,"Using existing formated files");
				return true;
			}
			
			if (pgm.equals("diamond")) formatdbCmd = BlastArgs.getDiamondFormat(dbFileName, dbFileName);
			else if (pgm.equals("usearch")) formatdbCmd = BlastArgs.getUsearchFormat(dbFileName, dbFileName);
			else {
				if (type.equals("nt")) formatdbCmd = BlastArgs.getFormatn(dbFileName);
				else formatdbCmd = BlastArgs.getFormatp(dbFileName);
			}
			CoreMain.PrtSpMsg(3,"Format database " + CoreMain.removeCurrentPath(dbFileName));
			long time = CoreMain.getTime();
			
			Process p = Runtime.getRuntime().exec(formatdbCmd);
			p.waitFor();
			if (p.exitValue() != 0) {
				CoreMain.PrtError(formatdbCmd + " failed with exit value = " + p.exitValue());
				return false;
			}
			CoreMain.PrtSpMsgTime(3, "Complete formatting, start search", time);
			return true;
		}
		catch (Exception e) {
			ErrorReport.reportError(e, "Running format database");
		}
		return true;
	}
	
	// remove if created by above routine
	private void removeFormatDB (String dbFilename, String[] rmFiles) {		
		try {
			//if (!CoreMain.doCleanup()) return;
			CoreMain.PrtSpMsg(2,"Removing format files " + dbFilename + ".*");
			for (int i = 0; i < rmFiles.length; i++) {
				String rmCmd = "rm " + dbFilename + rmFiles[i];
				if (debug) System.err.println(rmCmd);
				if (FileHelpers.fileExists(rmFiles[i])) {
					Process p = Runtime.getRuntime().exec(rmCmd);
					p.waitFor();
					if (p.exitValue() != 0) {
						CoreMain.PrtError("cleanup failed with exit value = " + p.exitValue());
					}
				}
			}
		}
		catch (Exception e) {
			ErrorReport.reportError(e, "Removing formatDB files");
		}
	}
	private boolean checkDiamondtmpDir(String blastOp) {
		if (!blastOp.contains("--tmpdir")) return true;
		String [] tok = blastOp.split(" ");
    		for (int i=0; i<tok.length; i++) {
    			if (!tok[i].equals("--tmpdir")) continue;
    			
			int j = i+1;
			if (j>=tok.length) {
				CoreMain.PrtError("diamond tmpdir option specified with no direcory argument");
				return false;
			}
			else {
				File d = new File(tok[j]);
				if (d.exists() && d.isDirectory()) return true;
				
			    try{
			        d.mkdir();
			        CoreMain.PrtSpMsg(3, "Created diamond tmpdir: " + tok[j]);
			        return true;
			     } catch(SecurityException e){
			        ErrorReport.prtReport(e, "Could not create diamond tmpdir: " + tok[j]);
			        return false;
			     }        
			}
    		}
    		return true;
	}
	// executed before first DB blast and then whenever there is a JPAVE_unitrans_subset_n parameter
	private boolean writeUniTransFile(boolean subset)
	{	
		File blastDir = new File(uniBlastDir);
		if (!blastDir.exists()) {
			CoreMain.PrtSpMsg(3,"Creating " + uniBlastDir + " directory");
			if (!blastDir.mkdir()) {
				System.err.println("*** Failed to create " + uniBlastDir + " directory " + blastDir);
				return false;
			}
		}
		if (subset) uniTransFile = uniBlastPrefix + "unitrans_sub.fasta";
		else {
			uniTransFile = uniBlastPrefix + "unitrans.fasta";
			if (FileHelpers.fileExists(uniTransFile)) return true;
		}
		
		if (subset) CoreMain.PrtSpMsg(3, "Create subset sequence file: " + uniTransFile);
		else CoreMain.PrtSpMsg(3, "Create sequence file: " + CoreMain.removeCurrentPath(uniTransFile));
		
		int cnt = sqlObj.writeUniTransFile(uniTransType, subset, new File(uniTransFile));
		if (cnt < 0) return false;
		CoreMain.PrtSpMsg(4, "Wrote " + cnt + " sequence records");		
		return true;
	}	
	
	/*
	 *  XXX read megablast or tblaxtx for the pairwise comparisons
	 *  TODO: these share the same pairsHash, so end up gathering HSPs for both
	 */
	public int getAllPairsFromBlastFile( 
			 HashMap <String, BlastHitData> pairsHash,
			 String fileBlast, boolean isSelf) throws Exception 
	{  	
		String strLine = null;
		String strHitContig, strQueryContig, keyR, key;
		BlastHitData hitData;
		boolean self=false, tself=false;
		if (isSelf) self=true; else tself=true;
		
		try {
			BufferedReader reader = CoreMain.openGZIP(fileBlast);

			int goodHits = 0, ignore = 0, cntRead = 0;

			while ((strLine = reader.readLine()) != null) {
				if (strLine.length() == 0 || strLine.charAt(0) == '#') continue; 

				cntRead++;
				String strFields[] = strLine.split("\t");
				strQueryContig = cleanID(strFields[0].trim());
				strHitContig = cleanID(strFields[1].trim());
				
				if (strQueryContig.equals(strHitContig)) continue; // Ignore self-hit
		
				// key must be same order as blast query-hit
				key = strQueryContig + ";" + strHitContig;
				keyR = strHitContig + ";" + strQueryContig;
						
      			if (pairsHash.containsKey(keyR)) {
       				hitData = pairsHash.get(keyR);
       				hitData.pairsAddHit(self, tself);
      			}
      			else if (pairsHash.containsKey(key)) {
       				hitData = pairsHash.get(key);
       				hitData.pairsAddHit(self, tself);
      			}
      			else {
      				hitData = new BlastHitData(false, strLine);
      				if (self) hitData.setIsSelf(true);
      				else	 hitData.setIsTself(true);
         			pairsHash.put(key, hitData);
         			goodHits++; 
      			}
 
				if (cntRead % 10000 == 0) 
					System.err.print("Load " + goodHits + " hits " + cntRead + " read ....\r");
			} 
			reader.close();
			if (ignore > 0) CoreMain.PrtWarn("Ignored " + ignore + " hits because e-value > 1e-20");
			return goodHits;
			
		} catch (Exception e) {
			CoreMain.PrtError("Input Line = '" + strLine + "'");
			ErrorReport.reportError(e, "Failed on reading all pairs from blast file.");
			return 0;
		} 
	}
	
	static private String cleanID(String str) {
		if (str.startsWith("lcl|"))
			return str.substring(4);
		else
			return str;
	}
	
	/**************************************************************************/
	public boolean doAnyBlast () {
		if (doSelfBlast) return true;
		if (doTSelfBlast) return true;
		for (int i=0; i< dbInfo.size(); i++) 
			if (dbInfo.get(i).doBlast) return true;
		return false;
	}

	public String getSelfBlastFile() {return selfBlastFile;}
	public String getTSelfBlastFile() {return tselfBlastFile;}
	
	public String getDBfastaFile(int i) {return dbInfo.get(i).DBfastaFile;}
	public String getDBfastaNoPath(int i) {return dbInfo.get(i).DBfastaNoPath;}
	public String getDBtype(int i) {return dbInfo.get(i).dbType;}
	public String getDBtaxo(int i) {return dbInfo.get(i).dbTaxo;}
	public boolean doSubset(int i) { return dbInfo.get(i).doSubset;}
	public String getDBparams(int i) { // for overview
			if (dbInfo.get(i).doBlast) {
				String a = dbInfo.get(i).blastArgs;
				if (a.contains("--tmpDir")) a = a.substring(0, a.indexOf("--tmpDir"));
				return dbInfo.get(i).searchPgm + "  " + a;
			}
			else {
				String a = dbInfo.get(i).tabularFile;
				if (a.contains("/")) a = a.substring(a.lastIndexOf("/")+1);
				return "Load " + a;
			}
	}
	// also in BlastHitData
	
	private void setType(int i, String t) {
		dbInfo.get(i).dbType = t;
		String capType = dbInfo.get(i).dbType.toUpperCase();
		String taxo = dbInfo.get(i).dbTaxo; 
		if (taxo.length() < 3) dbInfo.get(i).typeTaxo = capType + taxo;
		else dbInfo.get(i).typeTaxo = capType + dbInfo.get(i).dbTaxo.substring(0, 3);
	}
	public String getTypeTaxo(int i) {
		return dbInfo.get(i).typeTaxo;
	}
	public int getDbNum(int i) { return dbInfo.get(i).dbNum;}
	public boolean isProtein(int i) {return dbInfo.get(i).isProtein;}
	public String getDBdate(int i) {return dbInfo.get(i).dbDate;}
	
	public void setUniBlastPrefix(String p) {uniBlastPrefix = p;}
	public void setUniBlastDir(String p) {uniBlastDir = p;}
	public void setUniTransType(int t)   {uniTransType = t;}
	
	public void setSelfBlastFile(String s) {doSelfBlast=false; selfBlastFile=s;}
	public void setSelfBlastArgs(String s) {doSelfBlast=true; selfBlastArgs=s;}		
	public void setTSelfBlastFile(String s) {doTSelfBlast=false; tselfBlastFile=s;}
	public void setTSelfBlastArgs(String s) {doTSelfBlast=true;  tselfBlastArgs=s;}
	
	public void makeDB(int ix, String blast, String fasta, String args, String taxo, 
			String dbdate, boolean subset, String pgm) {
		DB d = new DB(ix, blast, fasta, args, taxo, dbdate, subset, pgm);
		dbInfo.add(d);
	}
	
	private DoUniProt uniObj = null;
	private CoreDBWrapper sqlObj = null;
	private CoreAnnotator annoObj = null;
	
	private String uniBlastPrefix = null; // uniBlast/<assemblyID> - check for complete blasts
	private String uniBlastDir = null;
	private String uniTransFile = null;	
	private int    uniTransType = 0;      // 0 - all, 1 = ctg only, 2 = singles only
	
	public boolean doSelfBlast = false;
    private String selfBlastFile = null;
    private String selfBlastArgs = null;
	
	public boolean doTSelfBlast = false;
	private String tselfBlastFile = null; // blast output file
	private String tselfBlastArgs = null;
			
	private boolean doDBblast = false;
	private ArrayList<DB> dbInfo = new ArrayList<DB>();
	
	private class DB {
		DB (int ix, String blast, String fasta, String args,  String taxo,  
				String dbdate, boolean subset, String pgm) {
			dbNum = ix;
			tabularFile = blast;
			DBfastaFile = fasta; // should exist even if there is a blast file -- for description & sequence
			DBfastaNoPath = fasta.substring(fasta.lastIndexOf("/")+1);
			blastArgs = args;  // should exist if no blast file, though may be default
			dbTaxo = taxo;
			doSubset = subset; // must be false if blastFile is present - checked in CoreMain
			dbDate = dbdate;
			if (!testExistString(tabularFile)) doBlast=true;
			else if (tabularFile.contains("_sub.tab")) doSubset=true; // previously run as subset, mark in pja_database 
			searchPgm=pgm;
		}
		int dbNum=0; 		// from PAVE.cfg '_n' 
		String tabularFile = "", DBfastaFile = "", blastArgs="";
		String dbType=""; // sp/tr/nt -- gets set on file check
		String dbTaxo="";   // plant, etc
		String typeTaxo="";
		String DBfastaNoPath="";
		String dbDate=null;
		boolean doSubset = false, isProtein = false, doBlast = false;
		String exec="blastx"; // blastn for nt -- computed in testDBfastaFile
		String searchPgm="";
		long fileSize=0;
	}
}
